#include	<algorithm>
#include	"utf16.h"
#include	<stdint.h>
#include	<iostream>
#include	<fstream>
#include	<vector>
using	namespace	std;
struct	Sort{
	uint16_t	i;
	uint64_t	n;
	bool	operator()(Sort	X,	Sort	Y){	return	X.n>Y.n;	}
};
int	main(int	ac,	char	**av){
	if(ac<2){	cerr<<"voca input [new_voca]\n";	return	0;	}
	Sort	vs[65536];	for(size_t	i=0;	i<65536;	i++){	vs[i].i=i;	vs[i].n=0;	}
	uint64_t	tot=0;	
	string	line;	vector<uint16_t>	u;
	ifstream	fi(av[1]);
	string	fn=av[1];	fn+=".utf16";
	FILE	*f=fopen(fn.c_str(),"wb");
	while(getline(fi,line)){
		line.push_back('\n');	u.resize(line.size());
		size_t	n=utf8_to_utf16((uint8_t*)line.data(),line.size(),u.data(),u.size());
		fwrite(u.data(),n*2,1,f);
		for(size_t	i=0;	i<n;	i++)	vs[u[i]].n++;
		tot+=n;
	}
	fclose(f);
	fi.close();
	if(ac>2){
	sort(vs,vs+65536,Sort());
	ofstream	fo("voca.txt");
	size_t	n=0;
	for(size_t	i=0;	i<65536;	i++)	if(vs[i].n>2){
		fo<<vs[i].i<<'\n';	n++;
	}
	fo.close();
	cerr<<"voca\t"<<(n/16)*16<<'\n';	
	}
	return	0;
}
